#include "TokenParser.h"
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <QColor>


TokenParser::TokenParser()
{

    m_keywords["and"] = ETokenAnd;
	m_keywords["break"] = ETokenBreak;
	m_keywords["do"] = ETokenDo;
	m_keywords["else"] = ETokenElse;
	m_keywords["elseif"] = ETokenElseif;
	m_keywords["end"] = ETokenEnd;
	m_keywords["false"] = ETokenFalse;
	m_keywords["for"] = ETokenFor;
	m_keywords["function"] = ETokenFunction;
	m_keywords["if"] = ETokenIf;
	m_keywords["in"] = ETokenIn;
	m_keywords["local"] = ETokenLocal;
	m_keywords["nil"] = ETokenNil;
	m_keywords["not"] = ETokenNot;
	m_keywords["or"] = ETokenOr;
	m_keywords["repeat"] = ETokenRepeat;
	m_keywords["return"] = ETokenReturn;
	m_keywords["then"] = ETokenThen;
	m_keywords["true"] = ETokenTrue;
	m_keywords["until"] = ETokenUntil;
	m_keywords["while"] = ETokenWhile;

    m_skipNewLines = false;
    m_skipComments = false;

}



TokenParser::~TokenParser()
{
	int count = m_tokens.size();
	for(int i = 0; i < count; i++)
		delete m_tokens[i];    
}

bool TokenParser::isCommentStart(const char * aData, int aOffset, int aDataLength, int & aCommentLength, int & aIndent)
{
	if(aDataLength - aOffset < 3)
		return false;
	if(aData[aOffset] != '-' || aData[aOffset + 1] != '-')
		return false;
	if(aDataLength - aOffset >= 4)//possible long comment start
	{
		if(aData[aOffset + 2] == '[')
		{
			int marker = aOffset + 3;
			int indent = 0;
			while(marker < aDataLength)
			{
				if(aData[marker] != '=')
					break;
				marker++;
				indent++;
			}
			if(marker == aDataLength)
				return false;
			if(aData[marker] == '[')
			{
				aCommentLength = marker - aOffset + 1;
				aIndent = indent;
				return true;
			}
		}
	}

	//this is one line comment then
	aCommentLength = 2;
	return true;

}


/*bool TokenParser::isLongBracketEnd(const char * aData, int aOffset, int aDataLength, int & aLength, int & aIndent)
{
	if(aDataLength - aOffset < 3)
		return false;
	if(aData[aOffset] != ']')
		return false;
	int marker = aOffset + 1;
	int indent = 0;
	while(marker < aDataLength)
	{
		if(aData[marker] != '=')
			break;
		marker++;
		indent++;
	}
	if(marker == aDataLength)
		return false;
	if(aData[marker] == ']')
	{
		aLength = marker - aOffset + 1;
		aIndent = indent;
		return true;
	}		
	return false;
}*/


bool TokenParser::isLongBracketStart(const char * aData, int aOffset, int aDataLength, int & aLength, int & aIndent)
{
	if(aDataLength - aOffset < 3)
		return false;
	if(aData[aOffset] != '[')
		return false;
	//printf("isLongBracketStart\n");
	int marker = aOffset + 1;
	int indent = 0;
	while(marker < aDataLength)
	{
		if(aData[marker] != '=')
			break;
		marker++;
		indent++;
	}
	if(marker >= aDataLength)
		return false;
	if(aData[marker] == '[')
	{
		aLength = marker - aOffset + 1;
		aIndent = indent;
		return true;
	}		
	return false;
}

bool isNumber(const char * aText)
{
	const char * ptr = aText;
	if(*ptr == 0)
		return false;
	while(*ptr)
	{
		if(*ptr < '0' || *ptr > '9')
			return false;
		ptr++;
	}
	return true;
}


void reallocBuffer(char ** aBuffer, int * oldLength, int newLength)
{
	if(newLength <= *oldLength)
		return;
	*oldLength = newLength*2;
	free(*aBuffer);
	*aBuffer = (char *)calloc(1, *oldLength + 1);
}

const int NOT_CLOSING_LONG_BRACKET = -1;
//this one operates on token level
//it returns -1 if m_tokens[aIndex] is not part of long bracket closing sequence
//otherwise a length of sequence is returned
//we can't just make a LuaLongBracketEnd token like it was done for LuaLongBracketStart, because things
//like test[a[2]] will be mislexed without context

int TokenParser::getClosingLongBracketLength(int aIndex)
{
    int count = m_tokens.size();
    if(aIndex >= count - 1)
        return NOT_CLOSING_LONG_BRACKET;//]] wouldn't fit in remaining tokens
    LuaToken * token = m_tokens[aIndex];
    if(token->getType() != ETokenCloseBracket)
        return NOT_CLOSING_LONG_BRACKET;//sequence doesn't start with ]
    //now count ='s of possible indent
    int marker = aIndex + 1;
    while(marker < count - 1 && m_tokens[marker]->getType() == ETokenEqualSign)
        marker++;
    token = m_tokens[marker];
    if(token->getType() != ETokenCloseBracket)
        return NOT_CLOSING_LONG_BRACKET;//sequence doesn't end with ]
    return marker - aIndex + 1;

}


void TokenParser::parseData(const char * aData)
{
	int marker = 0;
	int length = strlen(aData);
	bool isWord = false;
	m_tokens.clear();
	int bufferLength = 2048;
	char * buffer = (char *)calloc(1, bufferLength + 1);
	//printf("BUFFER %s\n", aData);
	for(int  i = 0; i < length; i++)
	{
		/*int character = aData[i];
        int cr = '\n';
        int lf = '\r';
        printf("%d (%c) vs %d vs %d\n", character, aData[i], cr, lf);*/
		if(IS_WORD_SYMBOL(aData[i]))
		{            
            if(!isWord)//word->non-word
			{
				isWord = true;
				marker = i;
			}
		}
		else
		{
			if(isWord)//non-word->word
			{
				isWord = false;
				int tokenLength = i - marker;
				if(tokenLength > 0)
				{
					char * text = (char *)calloc(tokenLength + 1, 1);
					memcpy(text, aData + marker, tokenLength);
					LuaToken * token = new LuaToken(text, marker, tokenLength);
					if(isNumber(text))
						token->setType(ETokenNumber);
                    else
                        token->setType(ETokenWord);
					free(text);
                    QString tokenText = token->getText();
                    if(m_keywords.contains(tokenText))
						token->setType(m_keywords[tokenText]);
					m_tokens.push_back(token);	
				}
				
			}
			if(!IS_WHITESPACE(aData[i]))
			{
				
				int indent = 0;
				int tokenLength = 0;

				if(isCommentStart(aData, i, length, tokenLength, indent))
				{					
					reallocBuffer(&buffer, &bufferLength, tokenLength);
					memset(buffer, 0, tokenLength + 1);
					memcpy(buffer, aData + i, tokenLength);
                    LuaToken * token = new LuaToken(QString(buffer), i, tokenLength);
					token->setIndent(indent);
					if(tokenLength == 2)
						token->setType(ETokenShortCommentStart);
					else
						token->setType(ETokenLongCommentStart);
					m_tokens.push_back(token);
                    i += tokenLength - 1;
				}
                /*else if(isLongBracketEnd(aData, i, length, tokenLength, indent))
				{
					reallocBuffer(&buffer, &bufferLength, tokenLength);
					memset(buffer, 0, tokenLength + 1);
					memcpy(buffer, aData + i, tokenLength);
                    Token * token = new Token(QString(buffer), i, tokenLength);
					token->setIndent(indent);					
					token->setType(ETokenLongBracketEnd);
					m_tokens.push_back(token);
                    i += tokenLength - 1;

                }*/
				else if(isLongBracketStart(aData, i, length, tokenLength, indent))
				{
					reallocBuffer(&buffer, &bufferLength, tokenLength);
					memset(buffer, 0, tokenLength + 1);
					memcpy(buffer, aData + i, tokenLength);
                    LuaToken * token = new LuaToken(QString(buffer), i, tokenLength);
					token->setIndent(indent);
					token->setType(ETokenLongBracketStart);				
					m_tokens.push_back(token);
                    i += tokenLength - 1;
				}
				else
				{
                    LuaToken * token = new LuaToken(QString(aData[i]), i, 1);
                    if(aData[i] == '\n' || aData[i] == '\r')
                    	token->setType(ETokenNewLine);
                    else if(aData[i] == '\"')
                    	token->setType(ETokenDoubleQuote);
                    else if(aData[i] == '\'')
                    	token->setType(ETokenSingleQuote);
                    else if(aData[i] == '\\')
                    	token->setType(ETokenSlash);
                    else if(aData[i] == ',')
                        token->setType(ETokenComma);
                    else if(aData[i] == '(')
                        token->setType(ETokenOpenParenthesis);
                    else if(aData[i] == ')')
                        token->setType(ETokenCloseParenthesis);
                    else if(aData[i] == '=')
                        token->setType(ETokenEqualSign);
                    else if(aData[i] == '.')
                        token->setType(ETokenDot);
                    else if(aData[i] == '[')
                        token->setType(ETokenOpenBracket);
                    else if(aData[i] == ']')
                        token->setType(ETokenCloseBracket);
                    else if(aData[i] == ':')
                        token->setType(ETokenColon);
                    else if(aData[i] == '{')
                        token->setType(ETokenOpenBrace);
                    else if(aData[i] == '}')
                        token->setType(ETokenCloseBrace);
					m_tokens.push_back(token);	
				}
				
			}
            else//whitespace
            {
                LuaToken * token = new LuaToken(QString(aData[i]), i, 1);
                token->setType(ETokenWhitespace);
                //printf("%d vs %d\n", (int)aData[i], '\t');
                if(aData[i] == '\t')
                	token->setTab(true);                 	
                m_tokens.push_back(token);
            }
		}
	}
	free(buffer);
	/*cout<<"Tokens:\n";
	for(int i = 0; i < m_tokens.size(); i++)
	{
		if(m_tokens[i]->isTab())
			cout<<"TAB"<<endl;
	}*/
    fillCommentsAndStrings();
    mergeWords();
	goToStart();
}

LuaToken * TokenParser::goToStart()
{
	m_position = 0;
	//find first non-whitespace token
	int count = m_tokens.size();
	for(int i = 0; i < count; i++)
	{
        if(m_tokens[i]->getType() != ETokenWhitespace
                && (m_skipNewLines == false || m_tokens[i]->getType() != ETokenNewLine)
                && (m_skipComments == false || m_tokens[i]->getType() != ETokenComment)
                )
		{
			m_position = i;
            return m_tokens[i];
		}	
	}
	return NULL;
}

LuaToken * TokenParser::goToEnd()
{
    m_position = 0;
    int count = m_tokens.size();
    for(int i = count - 1; i >= 0; i--)
    {
        if(m_tokens[i]->getType() != ETokenWhitespace
                && (m_skipNewLines == false || m_tokens[i]->getType() != ETokenNewLine)
                && (m_skipComments == false || m_tokens[i]->getType() != ETokenComment)
                )
        {
            m_position = i;
            return m_tokens[i];
        }
    }
    return NULL;
}

LuaToken * TokenParser::next()
{
	int initialPosition = m_position;

	int count = m_tokens.size();
    for(int i = initialPosition + 1; i < count; i++)
	{
		//skipping whitespace
        if(m_tokens[i]->getType() != ETokenWhitespace
                && (m_skipNewLines == false || m_tokens[i]->getType() != ETokenNewLine)
                && (m_skipComments == false || m_tokens[i]->getType() != ETokenComment)
                )
		{
			m_position = i;
			break;
		}
	}

	if(m_position != initialPosition)
	{
		return m_tokens[m_position];
	}
	m_position = initialPosition;
	return NULL;
	
}

LuaToken * TokenParser::previous()
{
	int initialPosition = m_position;
	
    for(int i = initialPosition - 1; i >= 0; i--)
	{
		//skipping whitespace
        if(m_tokens[i]->getType() != ETokenWhitespace
                && (m_skipNewLines == false || m_tokens[i]->getType() != ETokenNewLine)
                && (m_skipComments == false || m_tokens[i]->getType() != ETokenComment)
                )
		{
			m_position = i;
			break;
		}
	}

	if(m_position != initialPosition)
	{
		return m_tokens[m_position];
	}
	m_position = initialPosition;
	return NULL;
}

LuaToken * TokenParser::current()
{
    if(m_position < 0 || m_position >= (int)m_tokens.size())
		return NULL;
	return m_tokens[m_position];
}


void TokenParser::mergeSimilarTokens(vector<LuaToken *> & aMerged)
{
	LuaToken * previousToken = NULL;	
	int count = m_tokens.size();
	for(int i = 0; i < count; i++)
	{
		LuaToken * token = m_tokens[i];
		if(!previousToken)
		{
			aMerged.push_back(token);
			previousToken = token;
			continue;
		}
        if(previousToken->getType() != token->getType())
        {
			aMerged.push_back(token);
			previousToken = token;
        }
        else
        {
			//merge with previous
            previousToken->appendText(token->getText());
            int previousLength = previousToken->getLength();
            previousToken->setLength(previousLength + token->getLength());
        }
	}
	
}

//searches token of type aType starting from current position
LuaToken * TokenParser::findTokenOfType(TokenTypeEnum aType, int aIndent)
{
    int count = m_tokens.size();
    if(m_position >= count)
        return NULL;
    for(int i = m_position; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
            if(token->getType() == aType && token->getIndent() == aIndent)
                return token;
    }
    return NULL;
}


LuaToken * TokenParser::findLongBracketEnd(int aIndent)
{
    int count = m_tokens.size();
    if(m_position >= count)
        return NULL;
    for(int i = m_position; i < count - 1; i++)
    {
        LuaToken * token = m_tokens[i];
            if(token->getType() == ETokenCloseBracket && token->getIndent() == aIndent)
            {
                LuaToken * nextToken = m_tokens[i + 1];
                if(nextToken->getType() == ETokenCloseBracket && nextToken->getIndent() == aIndent)
                    return nextToken;
            }
    }
    return NULL;
}


void TokenParser::setTypeForRange(TokenTypeEnum aType, LuaToken * aStart, LuaToken * aEnd)
{
    bool inRange = false;
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        if(token == aStart)
            inRange = true;
        if(inRange)
        {
            token->setType(aType);
            token->setProcessed(true);
        }
        if(token == aEnd)
            inRange = false;
    }
}

/*void TokenParser::dumpHtml(TextSink * aSink)
{	
	vector<Token *>merged;
	mergeSimilarTokens(merged);
	int count = merged.size();   
    aSink->clearSink();
	for(int i = 0; i < count; i++)
	{
		Token * token = merged[i];
        LuaStyle * style = m_styles[token->getType()];
        if(style)
            aSink->appendHtml(style->formatToken(token->getText()));
        else
            aSink->appendPlainText(token->getText());
	}
}*/

//returns success
bool TokenParser::goToToken(LuaToken *aToken)
{
    if(m_tokens[m_position] == aToken)
        return true;
    //search forward first
    int count = m_tokens.size();
    for(int i = m_position; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        if(token == aToken)
        {
            m_position = i;
            return true;
        }
    }
    //not found try from the beginning then
    for(int i = 0; i < m_position; i++)
    {
        LuaToken * token = m_tokens[i];
        if(token == aToken)
        {
            m_position = i;
            return true;
        }
    }
    return false;
}


QVector<LuaToken *> & TokenParser::getTokens()
{
    return m_tokens;
}

bool TokenParser::needSuggestions(int aOffset)
{
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        int start = token->getStart();
        if(start <= aOffset && token->getLength() + start > aOffset)
        {
            if(token->getType() == ETokenComment || token->getType() == ETokenString)
                return false;
            else
                return true;
        }
    }
    //offset not found
    return false;
}

QString TokenParser::getModifiedWord(int aOffset, int aCharacter)
{
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        int start = token->getStart();
        if(start <= aOffset && token->getLength() + start >= aOffset)
        {
            //find a word part
            QString word = token->getText().left(aOffset - start);
            word += QChar(aCharacter);
            int lastNonWordCharacter = -1;
            int length = word.length();
            for(int t = 0; t < length; t++)
            {
                if(IS_WORD_SYMBOL(word[t]) == false && word[t] != '.')
                    lastNonWordCharacter = t;
            }
            if(lastNonWordCharacter >= 0)
                word = word.right(length - lastNonWordCharacter - 1);
            return word;
        }
    }
    return QString(QChar(aCharacter));//not found, shouldn't happen
}

void TokenParser::setSkipNewLines(bool aSkip)
{
    m_skipNewLines = aSkip;
}

void TokenParser::setSkipComments(bool aSkip)
{
    m_skipComments = aSkip;
}

bool TokenParser::getWordBoundaries(int aOffset, int &aStart, int &aLength)
{
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        int start = token->getStart();
        if(start <= aOffset && token->getLength() + start >= aOffset)
        {
            aStart = start;
            aLength = token->getLength();
            return true;
        }
    }
    return false;
}

LuaToken *TokenParser::getTokenByOffset(int aOffset)
{
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        int start = token->getStart();
        if(start <= aOffset && token->getLength() + start >= aOffset)
        {
            return token;
        }
    }
    return NULL;
}

void TokenParser::mergeWords()
{
    QVector<LuaToken *> mergedTokens;
    QVector<LuaToken *> toDelete;
    int count = m_tokens.size();
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        if(i > 0 && i < count - 2 && token->getType() == ETokenDot
                && m_tokens[i - 1]->getType() == ETokenWord
                && m_tokens[i + 1]->getType() == ETokenWord)
        {
            token->setType(ETokenWord);
        }
    }
    LuaToken * previousToken = NULL;
    for(int i = 0; i < count; i++)
    {
        LuaToken * token = m_tokens[i];
        if(!previousToken)
        {
            mergedTokens.push_back(token);
            previousToken = token;
            continue;
        }
        if(previousToken->getType() != token->getType() ||
                (token->getType() != ETokenWord && token->getType() != ETokenDot)
                || (previousToken->getType() != ETokenWord && previousToken->getType() != ETokenDot))
        {
            mergedTokens.push_back(token);
            previousToken = token;
        }
        else
        {
            //merge with previous
            previousToken->appendText(token->getText());
            int previousLength = previousToken->getLength();
            previousToken->setLength(previousLength + token->getLength());
            toDelete.push_back(token);
        }
    }
    m_tokens = mergedTokens;
    count = toDelete.size();
    for(int  i = 0; i < count; i++)
        delete toDelete[i];
}

const int MODE_NORMAL = 0;
const int MODE_SHORT_COMMENT = 1;
const int MODE_LONG_COMMENT = 2;
const int MODE_SHORT_STRING_QUOTE = 3;
const int MODE_SHORT_STRING_DOUBLE_QUOTE = 4;
const int MODE_LONG_STRING = 5;


TokenTypeEnum typeFromMode(int aMode)
{
    if(aMode == MODE_LONG_COMMENT)
        return ETokenComment;
    else if(aMode == MODE_SHORT_COMMENT)
        return ETokenComment;
    else if(aMode == MODE_LONG_STRING)
        return ETokenString;
    else if(aMode == MODE_SHORT_STRING_DOUBLE_QUOTE)
        return ETokenString;
    else if(aMode == MODE_SHORT_STRING_QUOTE)
        return ETokenString;
    return ETokenUnprocessed;
}


void TokenParser::fillCommentsAndStrings()
    {
        int count = m_tokens.size();
        bool isEscapeActivated = false;
        int mode = MODE_NORMAL;
        int indent = 0;
        TokenTypeEnum previousTokenType = ETokenUnprocessed;
        TokenTypeEnum prePreviousTokenType = ETokenUnprocessed;
        for(int i = 0; i < count; i++)
        {
            TokenTypeEnum tokenType = m_tokens[i]->getType();

            if(isEscapeActivated)
            {
                TokenTypeEnum modeType = typeFromMode(mode);
                if(modeType != ETokenUnprocessed)
                    m_tokens[i]->setType(modeType);
                isEscapeActivated = false;
                continue;
            }


            if(mode == MODE_NORMAL)
            {
                if(tokenType == ETokenLongCommentStart)
                {
                    mode = MODE_LONG_COMMENT;
                    indent = m_tokens[i]->getIndent();
                }
                else if (tokenType == ETokenShortCommentStart)
                    mode = MODE_SHORT_COMMENT;
                else if (tokenType == ETokenDoubleQuote)
                    mode = MODE_SHORT_STRING_DOUBLE_QUOTE;
                else if (tokenType == ETokenSingleQuote)
                    mode = MODE_SHORT_STRING_QUOTE;
                else if(tokenType == ETokenLongBracketStart)
                {
                    mode = MODE_LONG_STRING;
                    indent = m_tokens[i]->getIndent();
                }
                TokenTypeEnum modeType = typeFromMode(mode);
                if(modeType != ETokenUnprocessed)
                    m_tokens[i]->setType(modeType);
                continue;
            }
            //mode != MODE_NORMAL here
            //finish short comments
            if(tokenType == ETokenNewLine && mode == MODE_SHORT_COMMENT)
            {
                m_tokens[i]->setType(typeFromMode(mode));
                mode = MODE_NORMAL;
                continue;
            }

            int closingBracketLength = getClosingLongBracketLength(i);
            int closingBracketIndent = closingBracketLength - 2;
            if((mode == MODE_LONG_COMMENT || mode == MODE_LONG_STRING)
                && closingBracketLength != NOT_CLOSING_LONG_BRACKET
                && closingBracketIndent == indent)
            {
                for(int t = i; t < i + closingBracketLength; t++)
                {
                    m_tokens[t]->setType(typeFromMode(mode));
                }
                mode = MODE_NORMAL;
                i += closingBracketLength - 2;
                continue;
            }

            isEscapeActivated = tokenType == ETokenSlash;
            //end short strings with respect to escaping
            if(mode == MODE_SHORT_STRING_DOUBLE_QUOTE)
            {
                if(isEscapeActivated == false && (tokenType == ETokenDoubleQuote || tokenType == ETokenNewLine))
                {
                    m_tokens[i]->setType(typeFromMode(mode));
                    mode = MODE_NORMAL;
                    continue;
                }
            }
            if(mode == MODE_SHORT_STRING_QUOTE)
            {
                if(isEscapeActivated == false && (tokenType == ETokenSingleQuote || tokenType == ETokenNewLine))
                {
                    m_tokens[i]->setType(typeFromMode(mode));
                    mode = MODE_NORMAL;
                    continue;
                }
            }
            prePreviousTokenType = previousTokenType;
            previousTokenType = tokenType;
            TokenTypeEnum modeType = typeFromMode(mode);
            if(modeType != ETokenUnprocessed)
                m_tokens[i]->setType(modeType);
        }
    }
